#!/usr/bin/python

from __future__ import with_statement
from cStringIO import StringIO
#from contrib.sphinxapi import *
import sqlalchemy as sa
from Queue import Queue
from threading import Thread
import threading
from datetime import datetime, date
from time import strftime
import re, traceback, calendar, time, sys

#from local
from lib.database_qqshuqian import Database, g_pagesize, tt_init, tt_close, tt_get, tt_put
from lib.html_extractor import extract_text
from lib.utils import sanitizestring, transcode3, transcode5
from xml.sax.saxutils import escape
from lib.network import clean_url, fetch_url3

# type obj
class scheme:
    docid = 'seq'
    type = 'type'

csft_string_const = 'csft_string_const'
csft_string_fulltext = 'csft_string_fulltext'
csft_number = 'csft_number'
csft_float = 'csft_float'
csft_boolean = 'csft_boolean'
csft_timestamp = 'csft_timestamp'
csft_list = 'csft_list'
csft_file_pdf = 'csft_file_pdf'
csft_file_office = 'csft_file_office'

class QQShuqian:
    def __init__(self, O):
        #self.i = 0
        self.seq = -1
        self.content = ''
        #self.userid = None
        #self.articlenum = 0
        #self.done = True
        #self.uName = ''

        #self.tcinstance_txt = tt_init('192.168.1.103')
        self.tcinstance_txt = tt_init('localhost')
        self.dal = Database()

        self.cursor = 0
        self.workingset = []

        #for k in O.keys():
        #    print k, O[k] 


    def getScheme(self):
        return {'seq': {'seq': True},
                #'userid': {'type': csft_string_const}, 
                #'userid': {'type': csft_string_fulltext}, 
                #'articlenum': {'type': csft_number},
                #'done': {'type': csft_boolean},
                #'uName': {'type': csft_string_const},
                #'uName': {'type': csft_string_fulltext},
                'title': {'type': csft_string_fulltext},
                'content': {'type': csft_string_fulltext}}

    def Connected(self):
        pass

    def NextDocument(self):
        while True:
            barrier = len(self.workingset) if len(self.workingset) < g_pagesize else g_pagesize
            if self.cursor >= barrier:
                #self.workingset = self.dal._queryUser(self.i)
                self.workingset = self.dal._queryUser(self.seq)
                #self.workingset = [self.workingset[3]]
                #if self.i == 1:
                if not self.workingset:
                    return False
                else:
                    #self.i += 1
                    self.cursor = 0

            u = self.workingset[self.cursor]
            self.cursor += 1
            #print >>sys.stderr, u
            links = self.dal._queryUserLink(u[0])
            # if there are no bookmarks by user, let's forget it.
            if not links:
                continue

            #self.userid = u[0]
            #self.uName = u[1]
            #self.articlenum = u[2]
            #self.done = u[3]
            self.seq = int(u[5])
            self.content = ""
            for l in links:
                linkid = l[0]
                if False:
                    linkhtml = None
                    thislink = self.dal._queryOneLink(linkid)
                    try:
                        linkhtml = fetch_url3(clean_url(thislink[2]), retries = 2)
                    except Exception, e:
                        print >> sys.stderr, 'ERROR in FETCHING %s %s %s' % (linkid, thislink[2], thislink[1])
                        print >> sys.stderr, e
                else:
                    thislink = self.dal._queryOneLink(linkid)
                    title = thislink[1]
                    if not title:
                        continue
                    txt = tt_get(self.tcinstance_txt, linkid)
                    #self.content += ' '
                    #self.content += thislink[1]
                    if not txt or txt == '@@REDIRECT@@' or txt == '@@TINY@@' or txt == '@@EMPTY@@':
                        continue
                    self.content += ' '
                    self.content += title
                    self.content += ' '
                    self.content += txt
            if self.content == '':
                continue
            return True

    def OnBeforeIndex(self):
        pass

    def OnAfterIndex(self):
        pass

    def OnIndexFinished(self):
        self.dal = None
        tt_close(self.tcinstance_txt)


class QQShuqianLink:
    def __init__(self, O):
        self.seq = -1
        self.content = ''
        self.title = ''
        self.authorseq = -1

        #self.tcinstance_txt = tt_init('192.168.1.103')
        self.tcinstance_txt = tt_init('192.168.1.103')
        self.dal = Database()

        self.cursor = 0
        self.workingset = []

        #for k in O.keys():
        #    print k, O[k] 

    def getScheme(self):
        return {'seq': {'seq': True},
                'title': {'type': csft_string_fulltext},
                'content': {'type': csft_string_fulltext},
                'authorseq': {'type': csft_number}}

    def Connected(self):
        pass

    def NextDocument(self):
        while True:
            barrier = len(self.workingset) if len(self.workingset) < g_pagesize else g_pagesize
            if self.cursor >= barrier:
                self.workingset = self.dal._queryLink(self.seq)
                if not self.workingset:
                    return False
                else:
                    self.cursor = 0

            link = self.workingset[self.cursor]
            [linkid, title, url, time, obsolete, linkcount, authorid, seq] = link
            self.cursor += 1

            txt = tt_get(self.tcinstance_txt, linkid)
            self.seq = seq
            oneuser = self.dal._queryOneUser(authorid)
            if oneuser:
                self.authorseq = oneuser[5]
            else:
                print linkid + " " + str(seq) + " didn't find authorid " + authorid 
                continue
            self.title = title if title else " "
            self.content = txt if txt else " "
            if not txt or txt == '@@REDIRECT@@' or txt == '@@TINY@@' or txt == '@@EMPTY@@':
                continue
            return True

    def OnBeforeIndex(self):
        pass

    def OnAfterIndex(self):
        pass

    def OnIndexFinished(self):
        self.dal = None
        tt_close(self.tcinstance_txt)
